#!/bin/bash
# 作者:    		zhangyu@tt.com
# 创建时间:		2016/09/20
# 说明:			MySQL 自动巡检脚本（功能与oracle RDA脚本类似）
# 环境依赖:		MySQL 5.6版本及以上 + & sys schema
# 版本:			v0.2
# 依赖的包:		sysstat、MegaCli、lm_sensors、percona-toolkit

####################################变量定义################################

REPORTNAME="/checklog/report_$(hostname)_$(date '+%F').html";mkdir -p /checklog
MYUSR='xx'
MYPWD='xxx'
L_TITLE_FONT_SIZE=8
L1_TITLE_FONT_SIZE=6 #一级标题字号
L2_TITLE_FONT_SIZE=3  #二级标题字号
L3_TITLE_FONT_SIZE=2  #三级标题字号

####################################函数定义###############################
####################################功能函数
function echo3(){
echo
echo
echo
}

# 功能：右填充字符串
# 使用：echo "rpad 'OS info:'"
function rpad()
{
echo $1 | awk '{printf "%-20s\n",$0}'
}

# 功能：执行SQL语句
# 使用：my_executor "SQL语句"
function my_executor(){
\mysql -H -u${MYUSR} -p${MYPWD} sys -e "$1"
}

# 功能：获取MySQL参数值
# get_mysql_var "SQL语句"
function get_mysql_var(){
\mysql -s --skip-column-names -u${MYUSR} -p${MYPWD} -e "select variable_value from information_schema.global_variables where variable_name='$1'"
}

# 功能：添加html头标记
function html_begin(){
cat > ${REPORTNAME} <<EOF
<!DOCTYPE html>
<!--Ref1:http://www.w3school.com.cn/tags/tag_comment.asp-->
<!--Ref2:http://www.w3school.com.cn/tiy/t.asp?f=demo_html_textarea-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body>
EOF
}

# 功能：添加html尾标记
function html_end(){
cat >> ${REPORTNAME} <<EOF
</body>
</html>
EOF
}

# 功能：添加文本框
# 使用：add_textarea 行宽 列宽 "命令" 
function add_textarea(){
cat <<EOF
<textarea rows="$1" cols="$2">
EOF
eval "$3" 
echo "</textarea>"
}


# 功能：添加标题
# 使用：add_title 字号 "标题文本"
function add_title(){
  cat <<EOF
<p>
<font size="$1" face="Times">
<b>$2</b>
</font>
</p>
EOF
}

####################################数据收集函数:操作系统

function f_os_info(){
echo "$(rpad '操作系统版本:')"$(head -1 /etc/issue)
echo "$(rpad '内核信息:')"$(uname -a) 
echo3
echo "系统硬件信息:"
dmidecode -t system
}

function f_os_log(){
echo "DEMSG信息:"
dmesg | tail -10
echo3
echo "操作系统日志:"
tail -10 /var/log/messages
echo3
echo "操作系统日志(最近20条错误信息):"
egrep -i '(error|warn)' /var/log/messages | tail -20
echo3
echo "demsg日志(最近20条错误信息):"
dmesg | egrep -i '(error|warn)' | tail -20
echo3
echo "crond日志(最近20条错误信息):"
egrep -i '(error|warn)' /var/log/cron
}

function f_os_config(){
echo "CRONTAB信息(检查是否存在时间同步JOB):"
crontab -l | grep -v '^#'
echo3
echo "系统参数配置:"
sysctl -p 2>/dev/null | sort -d
echo3
echo "内核加载模块:"
lsmod | sort -d
echo3
echo "系统服务列表(确认crond进程为启动状态):"
chkconfig --list | sort -d
echo3
echo "PCI插槽信息:"
lspci -tv
}

function f_os_userinfo(){
echo "用户列表:"
cut -d: -f1 /etc/passwd | sort -d
echo3
echo "组列表:"
cut -d: -f1 /etc/group | sort -d
}

#CPU
function f_os_cpuinfo(){
echo "CPU型号:"
cat /proc/cpuinfo | grep -i 'model name' | uniq -c
echo3
echo "CPU 温度:"
sensors
echo3
echo "CPU 负载(最近5秒):"
mpstat 1 5
echo3
echo "进程数:"
ps -ef | wc -l
}

#Network
function f_os_networkinfo(){
echo "网络概况:"
ifconfig
echo3
echo "网卡信息:"
awk -F':' '{print $1}' /proc/net/dev  |\
egrep -v '(Inter-|face|lo)' |\
while read DEV;
  do  
   ethtool $DEV
done
echo3
echo "网络负载(最近5秒):"
sar -n DEV 1 5
echo3
echo "活动会话:"
w
echo3
echo "路由信息:"
route -n
echo3
echo "防火墙策略:"
iptables -L
#echo3
#echo "当前连接:"
#netstat -antp
}

#Memory
function f_os_memoryinfo(){
echo "内存概览:"
free -m
echo3
echo "vmstat 信息(最近5秒，主要关注so,si列):"
vmstat 1 5
echo3
echo "swappiness设置(数据库系统设置应该为0):"
cat /proc/sys/vm/swappiness
}

#Disk & filesystem
function f_os_vfsinfo(){
echo "物理磁盘信息:"
fdisk -l | grep Disk | grep -v 'Disk identifier'
echo3	
echo "磁盘使用率:"
df -hP | column -t
echo3
echo "逻辑卷信息:"
pvs;vgs;lvs	
echo3
echo "RAID卡配置信息(检查写策略是否为回写，否则查看BBU状态):"
/opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -Lall -aALL
echo3
echo "RAID其他信息:"
/opt/MegaRAID/MegaCli/MegaCli64 -fwtermlog -dsply -a0 | grep -i 'Measured chip temperature at Channel' | tail
/opt/MegaRAID/MegaCli/MegaCli64 -fwtermlog -dsply -a0 | grep -i temperature | grep -i 'Deg C'
/opt/MegaRAID/MegaCli/MegaCli64 -adpallinfo -a0 | grep FW
/opt/MegaRAID/MegaCli/MegaCli64 -pdlist -aall | grep 'Inquiry Data:'
/opt/MegaRAID/MegaCli/MegaCli64 -pdlist -aall | grep 'Firmware state:'
echo3
echo "fstab配置文件:"
cat /etc/fstab  | grep -v '^#'
echo3
echo "实际挂载磁盘信息:"
mount -l | column -t
echo3
echo "磁盘IO调度策略(确保使用deadline与noop):"
find /sys/block/* -name "[s|v]d[a-z]" |\
awk '{print $0,"/queue/scheduler"}' |\
sed 's/ //g' |\
xargs cat
echo3
echo "磁盘负载(最近5秒):"
iostat -xmt 1 5
echo3
#echo "根目录空间占用情况:"
#du -sm * 2>/dev/null | sort -n
#echo3
}


####################################数据收集函数:MySQL

f_mysql_pcn_advisor(){
pt-variable-advisor --user=${MYUSR} --password=${MYPWD} localhost
}

f_mysql_pcn_grantinfo(){
pt-show-grants --user=${MYUSR} --password=${MYPWD}
}

f_mysql_pcn_summary(){
pt-summary
}

f_mysql_pcn_mysqlsummary(){
pt-mysql-summary  --user=${MYUSR}  --password=${MYPWD} --databases
}

function f_mysql_errlog(){
add_title ${L3_TITLE_FONT_SIZE} "MySQL 错误日志:"
add_textarea 20 100 "tail -10000 $(get_mysql_var log_error) | egrep -i '(warn|error)' | tail -20"
}

function f_mysql_generalinfo(){
add_title ${L3_TITLE_FONT_SIZE} "MySQL版本信息:"
my_executor "select * from version"

add_title ${L3_TITLE_FONT_SIZE} "MySQL活动会话信息:"
my_executor "select * from session where command<>'Sleep'"
}

function f_mysql_initparams(){
echo "数据文件:"
echo "$(rpad '数据文件路径:')"$(get_mysql_var datadir)
echo "$(rpad 'innodb数据文件路径:')"$(get_mysql_var innodb_data_home_dir)
echo "$(rpad 'innodb数据文件:')"$(get_mysql_var innodb_data_file_path)
echo3
echo "binlog & innodb redolog:"
echo "$(rpad '是否开启binlog:')"$(get_mysql_var log_bin)
echo "$(rpad '当前SQL执行后是否记录binlog:')"$(get_mysql_var sql_log_bin)
echo "$(rpad 'binlog文件大小:')"$(get_mysql_var max_binlog_size)
echo "$(rpad 'binlog格式:')"$(get_mysql_var binlog_format)
echo "$(rpad 'binlog保留时长:')"$(get_mysql_var expire_logs_days)
echo3
echo "Innodb redolog信息:"
echo "$(rpad 'redolog路径:')"$(get_mysql_var innodb_log_group_home_dir)
echo "$(rpad 'redolog成员数:')"$(get_mysql_var innodb_log_files_in_group)
echo "$(rpad 'redolog大小:')"$(get_mysql_var innodb_log_file_size)
echo "$(rpad 'redolog共享池大小:')"$(get_mysql_var innodb_log_buffer_size)
echo "$(rpad 'redo刷新机制:')"$(get_mysql_var innodb_flush_log_at_trx_commit)
echo3
echo "缓存及IO信息:"
echo "$(rpad '共享缓存池大小:')"$(get_mysql_var innodb_buffer_pool_size)
echo "$(rpad 'IO刷新方式:')"$(get_mysql_var innodb_flush_method)
echo "$(rpad 'IO写线程并发数:')"$(get_mysql_var innodb_write_io_threads)
echo "$(rpad 'IO读线程并发数:')"$(get_mysql_var innodb_read_io_threads)
echo "$(rpad '查询缓存类型:')"$(get_mysql_var query_cache_type)
echo3
echo "资源限制参数:"
echo "$(rpad 'Innodb最大打开文件限制数:')"$(get_mysql_var innodb_open_files)
echo "$(rpad '最大表打开缓存限制数:')"$(get_mysql_var table_open_cache)
echo "$(rpad '最大连接数:')"$(get_mysql_var max_connections)
echo "$(rpad '最大可接受包大小(bytes):')"$(get_mysql_var max_allowed_packet)
echo3
echo "临时目录:"
echo "$(rpad '临时目录:')"$(get_mysql_var tmpdir)
echo3
echo "慢查询信息:"
echo "$(rpad '是否打开慢查询日志:')"$(get_mysql_var slow_query_log)
echo "$(rpad '慢查询阀值:')"$(get_mysql_var long_query_time)
echo3
echo "$(rpad '是否使用memory map:')"$(get_mysql_var myisam_use_mmap)
echo "$(rpad 'myisam恢复选项(建议为force,backup):')"$(get_mysql_var myisam_recover_options)
echo "$(rpad 'myisam延迟索引写(建议开启):')"$(get_mysql_var delay_key_write)
}

function f_mysql_innodbstats(){
\mysql -u${MYUSR} -p${MYPWD} <<EOF
show engine innodb status\G
EOF
}

function f_mysql_topsql(){

add_title ${L3_TITLE_FONT_SIZE} "排名前5 SQL：执行时发生全表扫描:"

my_executor "select 
    swfts.db,
    essbd.digest_text query,
    swfts.exec_count,
    swfts.rows_examined_avg,
    swfts.rows_sent,
    swfts.no_index_used_pct,
    swfts.total_latency,
    swfts.first_seen,
    swfts.last_seen,
    swfts.digest
from
    statements_with_full_table_scans swfts,
    performance_schema.events_statements_summary_by_digest essbd
where
    swfts.query = sys.format_statement(essbd.digest_text)
        and swfts.exec_count > 1000
        and swfts.rows_examined_avg > 100
        and swfts.last_seen between curdate() - interval 7 day and now()
order by rows_examined_avg desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5 SQL：按扫描行数:"

my_executor "
select 
    sa.db,
    essbd.digest_text,
    sa.exec_count,
    sa.rows_sent_avg,
    sa.rows_examined_avg,
    sa.digest,
    sa.first_seen,
    sa.last_seen
from
    statement_analysis sa,
    performance_schema.events_statements_summary_by_digest essbd
where
    sa.query = sys.format_statement(essbd.digest_text)
        and sa.rows_examined_avg > 100
        and sa.exec_count > 1000
        and sa.last_seen between curdate() - interval 7 day and now()
order by sa.rows_examined_avg desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5 SQL：更新删除行数:"

my_executor "
select 
    sa.db,
    essbd.digest_text,
    sa.exec_count,
    sa.rows_affected_avg,
    sa.digest,
    sa.first_seen,
    sa.last_seen
from
    statement_analysis sa,
    performance_schema.events_statements_summary_by_digest essbd
where
    sa.query = sys.format_statement(essbd.digest_text)
        and sa.rows_affected_avg > 20
        and sa.exec_count > 1000
        and sa.last_seen between curdate() - interval 7 day and now()
order by rows_affected_avg desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5 SQL：执行出现错误:"

my_executor "
select 
    sa.db,
    essbd.digest_text query,
    sa.exec_count,
    sa.err_count,
    sa.digest,
    sa.first_seen,
    sa.last_seen
from
    statement_analysis sa,
    performance_schema.events_statements_summary_by_digest essbd
where
    sa.query = sys.format_statement(essbd.digest_text)
        and sa.err_count > 0
        and sa.exec_count > 1000
        and sa.last_seen between curdate() - interval 7 day and now()
order by sa.err_count desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5 SQL：执行出现Warning:"

my_executor "
select 
    sa.db,
    essbd.digest_text,
    sa.exec_count,
    sa.warn_count,
    sa.digest,
    sa.first_seen,
    sa.last_seen
from
    statement_analysis sa,
    performance_schema.events_statements_summary_by_digest essbd
where
    sa.query = sys.format_statement(essbd.digest_text)
        and sa.warn_count > 0
        and sa.exec_count > 1000
        and sa.last_seen between curdate() - interval 7 day and now()
order by sa.warn_count desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5 SQL：执行期间生成磁盘临时表:"

my_executor "
select 
    sa.db,
    essbd.digest_text,
    sa.exec_count,
    sa.tmp_tables,
    sa.tmp_disk_tables,
    sa.digest,
    sa.first_seen,
    sa.last_seen
from
    statement_analysis sa,
    performance_schema.events_statements_summary_by_digest essbd
where
    sa.query = sys.format_statement(essbd.digest_text)
        and sa.tmp_disk_tables > 100
        and sa.exec_count > 1000
        and sa.last_seen between curdate() - interval 7 day and now()
order by tmp_disk_tables desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5 SQL：执行时发生排序操作:"

my_executor "
select 
    sa.db,
    essbd.digest_text,
    sa.exec_count,
    sa.rows_sorted,
    sa.sort_merge_passes,
    sa.digest,
    sa.first_seen,
    sa.last_seen
from
    statement_analysis sa,
    performance_schema.events_statements_summary_by_digest essbd
where
    sa.query = sys.format_statement(essbd.digest_text)
        and sa.sort_merge_passes > 0
        and sa.exec_count > 1000
        and sa.last_seen between curdate() - interval 7 day and now()
order by sort_merge_passes desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5 SQL：按延迟时间:"

my_executor "
select 
    sa.db,
    essbd.digest_text,
    sa.exec_count,
    sa.avg_latency,
    sa.lock_latency,
    sa.digest,
    sa.first_seen,
    sa.last_seen
from
    statement_analysis sa,
    performance_schema.events_statements_summary_by_digest essbd
where
    sa.query = sys.format_statement(essbd.digest_text)
        and sa.exec_count > 1000
        and sa.last_seen between curdate() - interval 7 day and now()
order by field(substring_index(avg_latency, ' ', - 1),
        'h',
        'm',
        's',
        'ms',
        'us',
        'ns',
        'ps'), substring_index(avg_latency, ' ', 1) desc
limit 5"
}

function f_mysql_summary(){
add_title ${L3_TITLE_FONT_SIZE} "排名前5：执行次数最多的clients:"
my_executor "select 
    *
from
    host_summary
order by statements desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5：执行次数最多的user:"
my_executor "select 
    *
from
    user_summary
order by statements desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "全局等待事件:"
my_executor "select 
    *
from
    waits_global_by_latency
order by total desc"

add_title ${L3_TITLE_FONT_SIZE} "排名前5：等待次数最多的host:"
my_executor "select 
    *
from
    waits_by_host_by_latency
order by total desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5：等待次数最多的user:"
my_executor "select 
    *
from
    waits_by_user_by_latency
order by total desc
limit 5"
}


function f_mysql_tableinfo(){

add_title ${L3_TITLE_FONT_SIZE} "排名前5：DML最频繁的表:"

my_executor "select 
    sts.table_schema,
    sts.table_name,
    sts.rows_inserted,
    sts.insert_latency,
    sts.rows_updated,
    sts.update_latency,
    sts.rows_deleted,
    sts.delete_latency,
    sts.rows_fetched,
    sts.fetch_latency
from
    schema_table_statistics sts
order by sts.rows_inserted + sts.rows_updated + sts.rows_deleted desc
limit 5"

add_title ${L3_TITLE_FONT_SIZE} "排名前5：全表扫描最频繁的表:"

my_executor "select 
    *
from
    schema_tables_with_full_table_scans
order by rows_full_scanned desc
limit 5"


add_title ${L3_TITLE_FONT_SIZE} "共享缓存池对象清单(database数据分布):"

my_executor "select 
    *
from
    innodb_buffer_stats_by_schema"

add_title ${L3_TITLE_FONT_SIZE} "数据库对象清单:"

my_executor "select 
    *
from
    schema_object_overview soo
where
    soo.db not in ('mysql' , 'performance_schema',
        'information_schema',
        'sys')
order by count desc"

add_title ${L3_TITLE_FONT_SIZE} "大小超过1GB(包含索引)的表清单:"

my_executor "select 
    table_schema,
    table_name,
    round(sum(data_length + index_length) / power(1024, 3),
            2) gbs,
    sum(table_rows) table_rows,
    count(*) cnt
from
    information_schema.tables
group by table_schema , table_name
having sum(data_length + index_length) / power(1024, 3) >= 1
order by gbs desc
limit 20"


}

function f_mysql_indexinfo(){

add_title ${L3_TITLE_FONT_SIZE} "排名前10索引：按DML次数:"

my_executor "select 
    sis.*,
    round((rows_selected / (rows_inserted + rows_updated + rows_deleted + rows_selected) * 100)) rows_selected_pct
from
    schema_index_statistics sis
where
    rows_inserted + rows_updated + rows_deleted + rows_selected > 10000
order by rows_inserted + rows_updated + rows_deleted desc
limit 10"

add_title ${L3_TITLE_FONT_SIZE} "冗余的索引:"

my_executor "select 
    *
from
    schema_redundant_indexes"

add_title ${L3_TITLE_FONT_SIZE} "未使用的索引:"

my_executor "select 
    object_schema, count(*) cnt
from
    schema_unused_indexes sui
group by object_schema
order by 1"
}

function f_mysql_usage(){

add_title ${L3_TITLE_FONT_SIZE} "排名前5：自增列使用情况:"

my_executor "select 
    *
from
    schema_auto_increment_columns
order by auto_increment_ratio desc
limit 5"
}

function f_mysql_iostat(){

add_title ${L3_TITLE_FONT_SIZE} "文件读写：按字节排序（分类）:"

my_executor "select * from io_global_by_wait_by_bytes"

add_title ${L3_TITLE_FONT_SIZE} "文件读写：按延迟排序（分类）:"

my_executor "select * from io_global_by_wait_by_latency"

add_title ${L3_TITLE_FONT_SIZE} "排名前10：文件读写次数:"

my_executor "select * from io_global_by_file_by_bytes order by count_read+count_write desc limit 10"

add_title ${L3_TITLE_FONT_SIZE} "排名前10：文件读写延迟（按thread）:"

my_executor "select * from io_by_thread_by_latency order by total desc limit 10"
}

####################################拼接HTML巡检报告###############################
:<<CMMT
函数列表：

	操作系统：
		f_os_info
		f_os_cpuinfo
		f_os_memoryinfo	
        f_os_vfsinfo
		f_os_networkinfo

		f_os_config
		f_os_log

		f_os_userinfo

	MySQL：
		f_mysql_errlog	
		f_mysql_initparams
		f_mysql_innodbstats		
		f_mysql_generalinfo
		f_mysql_topsql
		f_mysql_summary
		f_mysql_tableinfo
		f_mysql_indexinfo
		f_mysql_usage
		f_mysql_iostat
CMMT

html_begin
add_title ${L_TITLE_FONT_SIZE} "$(hostname) 巡检日志($(date '+%F'))" >> ${REPORTNAME}

##################OS Info

add_title ${L1_TITLE_FONT_SIZE} "操作系统信息" >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "操作系统版本:" >> ${REPORTNAME}
add_textarea 5 100 f_os_info >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "CPU信息:" >> ${REPORTNAME}
add_textarea 20 100 f_os_cpuinfo >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "内存信息:" >> ${REPORTNAME}
add_textarea 20 100 f_os_memoryinfo >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "IO及文件系统信息:" >> ${REPORTNAME}
add_textarea 20 100 f_os_vfsinfo >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "网络系统:" >> ${REPORTNAME}
add_textarea 20 100 f_os_networkinfo >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "操作系统配置(service及内核参数等):" >> ${REPORTNAME}
add_textarea 20 100 f_os_config >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "操作系统日志(demsg与messages):" >> ${REPORTNAME}
add_textarea 20 100 f_os_log >> ${REPORTNAME}

##################MySQL Info

add_title ${L1_TITLE_FONT_SIZE} "MySQL信息" >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【一般信息】" >> ${REPORTNAME}
f_mysql_generalinfo >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【初始化参数】" >> ${REPORTNAME}
add_textarea 20 100 f_mysql_initparams >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【TOP 客户端&用户】" >> ${REPORTNAME}
f_mysql_summary >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【IO信息】" >> ${REPORTNAME}
f_mysql_iostat >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【错误日志】" >> ${REPORTNAME}
f_mysql_errlog >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【自增列使用情况】" >> ${REPORTNAME}
f_mysql_usage >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【表详细信息】" >> ${REPORTNAME}
f_mysql_tableinfo >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【索引详细信息】" >> ${REPORTNAME}
f_mysql_indexinfo >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【TOP SQL】" >> ${REPORTNAME}
f_mysql_topsql >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【Innodb存储引擎详细信息】" >> ${REPORTNAME}

add_textarea 30 100 f_mysql_innodbstats >> ${REPORTNAME}

add_title ${L2_TITLE_FONT_SIZE} "【Percona工具集信息】" >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "MySQL 顾问反馈信息:" >> ${REPORTNAME}
add_textarea 20 100 f_mysql_pcn_advisor >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "MySQL 服务器汇总信息:" >> ${REPORTNAME}
add_textarea 20 100 f_mysql_pcn_summary >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "MySQL 数据库汇总信息:" >> ${REPORTNAME}
add_textarea 20 100 f_mysql_pcn_mysqlsummary >> ${REPORTNAME}

add_title ${L3_TITLE_FONT_SIZE} "MySQL 权限信息:" >> ${REPORTNAME}
add_textarea 20 100 f_mysql_pcn_grantinfo >> ${REPORTNAME}

html_end